from  sgmllib import SGMLParser,SGMLParseError 
import re
_delPat=re.compile('<!.+?>|<script.+?</script>|<style.+?</style>', \
				re.S|re.I)
def html_tidy(strHTML):
	return _delPat.sub('',strHTML)

class BaseHTMLParser(SGMLParser):

	
	def __init__(self):
		self._anchorLinks=[]
		self.__anchorTexts=[]
		self._imageLinks=[]		
		self.__frameLinks=[]
		self.__contents=[]

		self.__inA=False
		self.__curAnchorText=''
		self.__curAnchorLink=''
		self.__content=''
		
		
		SGMLParser.__init__(self)
	
		
	
		
		
	def reset(self):
		SGMLParser.reset(self)
		self._anchorLinks=[]
		self.__anchorTexts=[]
		self._imageLinks=[]		
		self.__frameLinks=[]
		self.__contents=[]
		self.__inA=False
		self.__curAnchorText=''
		self.__curAnchorLink=''
		self.__content=''
	def feed(self,data,tidy=False):
		#you should put all data in the html page once
		if tidy:
			data=html_tidy(data)
		SGMLParser.feed(self,data)
		#force feed all data 
		self.close()
		self.__content=''.join(self.__contents)
	def start_a(self,attrs):
		#print 'start_a'
		for attr,val in attrs:
			if attr=='href':
				self._anchorLinks.append(val)
				self.__curAnchorLink=val
				self.__inA=True
	def handle_data(self,data):
		data=data.strip()
		if data:
			self.__contents.append(data+'\n')
		if self.__inA:
			self.__curAnchorText+=data	
	def end_a(self):
		if self.__inA:
			self.__anchorTexts.append((self.__curAnchorLink,self.__curAnchorText))
			self.__curAnchorText=''
		self.__inA=False
	def start_frame(self,attrs):
		for attr,val in attrs:
			if attr=='src':
				self.__frameLinks.append(val)
	

	def start_link(self,attrs):
		for attr,val in attrs:
			if attr=='href':
				self._anchorLinks.append(val)
			elif attr=='':
				pass
	def start_img(self,attrs):
		for attr,val in attrs:
			if attr=='src':
				self._imageLinks.append(val)
				break
	def anchor_links(self):
		return self._anchorLinks
	def anchor_links_with_text(self):
		return self.__anchorTexts
	def image_links(self):
		return self._imageLinks
	def links(self):
		return self.anchor_links()+self.__frameLinks+self.image_links()
	def content(self):
		return self.__content
	
		
def test():
	p=BaseHTMLParser()
	#strHTML=open('test.html','r').read()
	import urllib
	#strHTML=urllib.urlopen('http://net.pku.edu.cn/~webg').read()
	strHTML=urllib.urlopen('http://net.pku.edu.cn/~webg/TianwangLiterature/Bachelor%20Thesis/').read()
	
	#print strHTML
	print '------------------------------' 
	p.feed(strHTML)
	print '------------------------------' 
	#print p.href_links()
	#print p.anchor_texts()
	from URL import URL
	
	assert len(p.links())==77
	for l in p.links():
		u=URL(l,baseURL='http://net.pku.edu.cn/~webg/TianwangLiterature/Bachelor%20Thesis/')
		print u.full_url(withFrag=False)
	p.reset()
	p.feed(strHTML)
	assert len(p.links())==77
	#p.reset()
	p.feed(strHTML)
	print len(p.links())
	assert len(p.links())==154
	'''
	print '********************************'
	from HarvestManParser import BaseHTMLParser as HParser
	p2=HParser()
	strHTML=p._tidy(strHTML)
	p2.feed(strHTML)
	print p2.get_links()
	'''

def test2():
	p=BaseHTMLParser()
	#strHTML=open('test.html','r').read()
	from urllib import urlopen
	#strHTML=urllib.urlopen('http://net.pku.edu.cn/~webg').read()
	#strHTML=urllib.urlopen('http://blog.sina.com.cn/m/free_mark_vision_studio').read()
	#strHTML=urlopen('http://blog.sina.com.cn/m/zuoluogufeiyang').read()
	strHTML=urlopen('http://net.pku.edu.cn').read()
	#print strHTML
	print '------------------------------' 
	p.feed(strHTML,tidy=True)
	print '------------------------------' 
	#print p.href_links()
	#print p.anchor_texts()
	print p.content()	
	
	
if __name__=='__main__':
	test2()
